# -*- coding: utf-8 -*-
"""
Created on Mon Aug  7 09:17:39 2023

@author: tyshixi08
"""

import pandas as pd
import numpy as np
import statsmodels.api as sm
import time
import datetime
from datetime import date
import os
from tqdm import tqdm

def factor_pre(df):
    """
    因子值预处理
    """
    def extreme_3sigma(df_slice, n=3):
        # 去异常值：3sigma法去极值
        if abs(df_slice['value']).sum() != 0:
            mean = np.nanmean(df_slice['value'])  # 截面数据均值
            std = np.nanstd(df_slice['value'])  # 截面数据标准差
            dt_up = mean + n * std  # 上限
            dt_down = mean - n * std  # 下限
            df_slice['value'] = df_slice['value'].clip(dt_down, dt_up, axis=0)  # 超出上下限的值，赋值为上下限
        else:
            df_slice = df_slice
        return df_slice

    def standardize_z(df_slice):
        # ZScore标准化
        if abs(df_slice['value']).sum() != 0:
            mean = np.nanmean(df_slice['value'], axis=0)  # 截面数据均值
            std = np.nanstd(df_slice['value'], axis=0)  # 截面数据标准差
            df_slice['value'] = (df_slice['value'] - mean) / std
        else:
            df_slice = df_slice
            print('该因子{}数据均为0'.format(df_slice.t_date.unique()))
        return df_slice

    tqdm.pandas(desc='apply')
    print('ZScore标准化:')
    df = df.groupby('t_date').progress_apply(standardize_z).reset_index(drop=True)
    print('3sigma法去极值:')
    df = df.groupby('t_date').progress_apply(extreme_3sigma).reset_index(drop=True)
    return df


class ConvFactorCalculation(object):
    def __init__(self, start, end):
        df_after = pd.read_pickle(os.path.join('data', 'data_after_filter.pkl'))
        df_after = df_after.sort_values(['datetime', 'c_bondCode']).reset_index(drop=True)
        df_after = df_after[(df_after['Date'] >= start) & (df_after['Date'] <= end)]
        if len(df_after) == 0:
            self.df_after = df_after
        else:
            df_valuation_stock = pd.read_pickle(os.path.join('data', 'data_all.pkl'))
            df_valuation_stock = df_valuation_stock.sort_values(['datetime', 'c_bondCode']).reset_index(drop=True)
            start1 = (pd.to_datetime(start) - datetime.timedelta(days=180)).strftime('%Y-%m-%d')
            df_valuation_stock = df_valuation_stock[(df_valuation_stock['Date'] >= start1) &
                                                    (df_valuation_stock['Date'] <= end)]
    
            self.df_after = df_after
            self.df_valuation_stock = df_valuation_stock
            self.start = start
            self.end = end
            self.n_trade = len(df_after[df_after['Date'] == df_after['Date'].values[0]]['datetime'].unique())  # 一天交易的次数
    
    def judgement(self):
        df = self.df_after
        return len(df)

    def sec_bond_premium_5m(self):
        # 纯债溢价率 sec_bond_premium_5m
        df = self.df_after[['Date', 'datetime', 'c_bondCode', 'n_bondValue', 'bond_close']].copy()
        df['sec_bond_premium_5m'] = df['bond_close'] / df['n_bondValue'] - 1
        df = df.rename(columns={'datetime': 't_date', 'c_bondCode': 'c_code', 'sec_bond_premium_5m': 'value'})
        return factor_pre(df[['t_date', 'c_code', 'value']])

    def ts_bond_premium_5m(self, ND):
        # 纯债溢价率分位数 ts_bond_premium_rollNDtdays_5m
        n_trade = self.n_trade
        df = self.df_after[['Date', 'datetime', 'c_bondCode', 'n_bondValue', 'bond_close']].copy()
        df['sec_bond_premium_5m'] = df['bond_close'] / df['n_bondValue'] - 1
        df_valuation_stock = self.df_valuation_stock[
            ['Date', 'datetime', 'c_bondCode', 'n_bondValue', 'bond_close']].copy()
        df_valuation_stock['sec_bond_premium_5m'] = df_valuation_stock['bond_close'] /\
                                                    df_valuation_stock['n_bondValue'] - 1

        df_valuation_stock = df_valuation_stock.set_index('datetime')
        # 滚动ND * n_trade个数据点计算
        min_max_bp = df_valuation_stock.groupby('c_bondCode').rolling(ND * n_trade).agg(
            {'sec_bond_premium_5m': ['min', 'max']}).reset_index()
        min_max_bp.columns = ['c_bondCode', 'datetime', 'min_sec_bond_premium_5m', 'max_sec_bond_premium_5m']
        df = pd.merge(df, min_max_bp, on=['c_bondCode', 'datetime'], how='left')
        df['ts_bond_premium_roll{}tdays_5m'.format(ND)] = (df['sec_bond_premium_5m'] - df[
            'min_sec_bond_premium_5m']) / (df['max_sec_bond_premium_5m'] - df['min_sec_bond_premium_5m'])

        df = df.rename(columns={'datetime': 't_date', 'c_bondCode': 'c_code',
                                'ts_bond_premium_roll{}tdays_5m'.format(ND): 'value'})
        return factor_pre(df[['t_date', 'c_code', 'value']])

    def sec_price_5m(self):
        # 转债价格 sec_price_5m
        df = self.df_after[['Date', 'datetime', 'c_bondCode', 'bond_close']].copy()
        df = df.rename(columns={'datetime': 't_date', 'c_bondCode': 'c_code', 'bond_close': 'value'})
        return factor_pre(df[['t_date', 'c_code', 'value']])

    def ts_price_5m(self, ND):
        # 转债价格分位数 ts_price_rollNDtdays_5m
        n_trade = self.n_trade
        df = self.df_after[['Date', 'datetime', 'c_bondCode', 'bond_close']].copy()
        df_valuation_stock = self.df_valuation_stock[['Date', 'datetime', 'c_bondCode', 'bond_close']].copy()

        df_valuation_stock = df_valuation_stock.set_index('datetime')
        # 滚动ND * n_trade个数据点计算
        min_max_cp = df_valuation_stock.groupby('c_bondCode').rolling(ND * n_trade).agg(
            {'bond_close': ['min', 'max']}).reset_index()
        min_max_cp.columns = ['c_bondCode', 'datetime', 'min_sec_price_5m', 'max_sec_price_5m']
        df = pd.merge(df, min_max_cp, on=['c_bondCode', 'datetime'], how='left')
        df['ts_price_roll{}tdays_5m'.format(ND)] = (df['bond_close'] - df['min_sec_price_5m']) / (
                    df['max_sec_price_5m'] - df['min_sec_price_5m'])

        df = df.rename(columns={'datetime': 't_date', 'c_bondCode': 'c_code',
                                'ts_price_roll{}tdays_5m'.format(ND): 'value'})
        return factor_pre(df )

    def sec_price_skew_5m(self, MD):
        # M=5 转债价格偏度 ts_price_skewNDd_5m
        n_trade = self.n_trade
        df = self.df_after[['Date', 'datetime', 'c_bondCode', 'bond_close', 'volume']].copy()
        df_valuation_stock = self.df_valuation_stock[['Date', 'datetime', 'c_bondCode', 'bond_close', 'volume']].copy()

        df_valuation_stock = df_valuation_stock.set_index('datetime')
        # 滚动ND * n_trade个数据点计算
        close_skew = df_valuation_stock.groupby('c_bondCode').rolling(MD * n_trade).agg(
            {'bond_close': ['mean', 'std'], 'volume': 'sum'}).reset_index()
        close_skew.columns = ['c_bondCode', 'datetime', 'close_mean', 'close_std', 'volume_sum']
        
        def cal_skew(sub):
            sub = sub.tail(MD * n_trade)
            sub_mean = sub['close_mean'].iloc[-1]
            sub_std = sub['close_std'].iloc[-1]
            sub_sum = sub['volume_sum'].iloc[-1]
            sub_skew = (sub['volume'] / sub_sum) * ((sub['bond_close'] - sub_mean) ** 3 / sub_std ** 3)
            sub_skew = sub_skew.sum()
            return sub_skew
        
        df_valuation_stock = pd.merge(df_valuation_stock, close_skew, on=['c_bondCode', 'datetime'], how='right')
        df_skew = []
        for i in tqdm(df['datetime'].unique()):
            skew_i = df_valuation_stock[df_valuation_stock.datetime <= i].groupby('c_bondCode').apply(cal_skew)
            skew_i.name = i
            df_skew.append(skew_i)
            
        df_skew = pd.concat(df_skew,axis=1).stack().reset_index()
        df_skew.columns = ['c_bondCode','datetime','sec_price_skew{}d_5m'.format(MD)]
        df = df.merge(df_skew, left_on=['c_bondCode','datetime'], right_on=['c_bondCode','datetime'], how='left')

        df = df.rename(columns={'datetime': 't_date', 'c_bondCode': 'c_code', 'sec_price_skew{}d_5m'.format(MD): 'value'})
        return factor_pre(df[['t_date', 'c_code', 'value']])

    def sec_turnover_5m(self):
        # 换手率 sec_turnover_5m
        df = self.df_after[['Date', 'datetime', 'c_bondCode', 'total_turnover', 'n_outstandingBalance']].copy()
        df['sec_turnover_5m'] = df['total_turnover'] / df['n_outstandingBalance'] / 10 ** 8
        df = df.rename(columns={'datetime': 't_date', 'c_bondCode': 'c_code', 'sec_turnover_5m': 'value'})
        return factor_pre(df[['t_date', 'c_code', 'value']])

    def ts_turnover_5m(self, ND):
        # 换手率分位数 ts_turnover_rollNDtdays_5m
        n_trade = self.n_trade
        df = self.df_after[['Date', 'datetime', 'c_bondCode', 'total_turnover', 'n_outstandingBalance']].copy()
        df['sec_turnover_5m'] = df['total_turnover'] / df['n_outstandingBalance'] / 10 ** 8
        df_valuation_stock = self.df_valuation_stock[
            ['Date', 'datetime', 'c_bondCode', 'total_turnover', 'n_outstandingBalance']].copy()
        df_valuation_stock['sec_turnover_5m'] = df_valuation_stock['total_turnover'] / df_valuation_stock[
            'n_outstandingBalance'] / 10 ** 8

        df_valuation_stock = df_valuation_stock.set_index('datetime')
        # 滚动ND * n_trade个数据点计算
        min_max_cc = df_valuation_stock.groupby('c_bondCode').rolling(ND * n_trade).agg(
            {'sec_turnover_5m': ['min', 'max']}).reset_index()
        min_max_cc.columns = ['c_bondCode', 'datetime', 'min_sec_turnover_5m', 'max_sec_turnover_5m']
        df = pd.merge(df, min_max_cc, on=['c_bondCode', 'datetime'], how='left')
        df['ts_turnover_roll{}tdays_5m'.format(ND)] = (df['sec_turnover_5m'] - df['min_sec_turnover_5m']) / (
                    df['max_sec_turnover_5m'] - df['min_sec_turnover_5m'])

        df = df.rename(columns={'datetime': 't_date', 'c_bondCode': 'c_code',
                                'ts_turnover_roll{}tdays_5m'.format(ND): 'value'})
        return factor_pre(df[['t_date', 'c_code', 'value']])

    def sec_volatility_5m(self, MD):
        # M=5 波动率 sec_volatilityMDd
        n_trade = self.n_trade
        df = self.df_after[['Date', 'datetime', 'c_bondCode', 'bond_pctChange_5m']].copy()
        df_valuation_stock = self.df_valuation_stock[['Date', 'datetime', 'c_bondCode', 'bond_pctChange_5m']].copy()

        df_valuation_stock = df_valuation_stock.set_index('datetime')
        # 滚动MD * n_trade个数据点计算
        vol = df_valuation_stock.groupby('c_bondCode').rolling(int(MD * n_trade)).bond_pctChange_5m.std().reset_index()
        vol.columns = ['c_bondCode', 'datetime', 'sec_volatility{}d_5m'.format(MD)]
        df = pd.merge(df, vol, on=['c_bondCode', 'datetime'], how='left')

        df = df.rename(columns={'datetime': 't_date', 'c_bondCode': 'c_code', 'sec_volatility{}d_5m'.format(MD): 'value'})
        return factor_pre(df[['t_date', 'c_code', 'value']])

    def ts_volatility_5m(self, MD, ND):
        # M=1/4, ND=1 波动率分位数 ts_volatilityMDd_rollNDtdays_5m
        n_trade = self.n_trade
        df = self.df_after[['Date', 'datetime', 'c_bondCode', 'bond_pctChange_5m']].copy()
        df_valuation_stock = self.df_valuation_stock[['Date', 'datetime', 'c_bondCode', 'bond_pctChange_5m']].copy()

        df_valuation_stock = df_valuation_stock.set_index('datetime')
        # 滚动ND * n_trade个数据点计算
        vol = df_valuation_stock.groupby('c_bondCode').rolling(int(MD*(n_trade-2))).bond_pctChange_5m.std().reset_index()
        vol.columns = ['c_bondCode', 'datetime', 'sec_volatility{}h'.format(int(MD*4))]
        df_valuation_stock = df_valuation_stock.reset_index()
        df_valuation_stock = pd.merge(df_valuation_stock, vol, on=['c_bondCode', 'datetime'], how='left')
        df = pd.merge(df, vol, on=['c_bondCode', 'datetime'], how='left')

        df_valuation_stock = df_valuation_stock.set_index('datetime')
        # 滚动MD * n_trade个数据点计算
        min_max_v = df_valuation_stock.groupby('c_bondCode').rolling(ND * n_trade).agg({'sec_volatility{}h'.format(int(MD*4)): ['min', 'max']}).reset_index()
        min_max_v.columns = ['c_bondCode', 'datetime', 'min_volatility_5m', 'max_volatility_5m']
        df = pd.merge(df, min_max_v, on=['c_bondCode', 'datetime'], how='left')
        df['ts_volatility{}h_roll{}tdays_5m'.format(int(MD*4), ND)] = (df['sec_volatility{}h'.format(int(MD*4))] - df['min_volatility_5m']) / (df['max_volatility_5m'] - df['min_volatility_5m'])
        df = df.rename(columns={'datetime': 't_date', 'c_bondCode': 'c_code', 'ts_volatility{}h_roll{}tdays_5m'.format(int(MD*4), ND): 'value'})
        return factor_pre(df[['t_date', 'c_code', 'value']])

    def sec_amplitude_5m(self):
        # 振幅 sec_amplitude_5m
        def cal_adjust(df_slice):
            df_slice = df_slice.copy()
            df_slice['sec_amplitude_5m'][df_slice['tail'] == '09:25:00'] = np.nan
            df_slice = df_slice.sort_values(['datetime'])
            df_slice = df_slice.fillna(method = 'ffill')
            return df_slice
        
        df = self.df_after[['Date', 'datetime', 'c_bondCode', 'bond_close', 'bond_high', 'bond_low', 'bond_pctChange_5m']].copy()
        df_valuation_stock = self.df_valuation_stock[['Date', 'datetime', 'c_bondCode', 'bond_close', 'bond_high', 'bond_low', 'bond_pctChange_5m','tail']].copy()
        df_valuation_stock['sec_amplitude_5m'] = (df_valuation_stock['bond_high'] - df_valuation_stock['bond_low']) / (df_valuation_stock['bond_close'] / (1 + df_valuation_stock['bond_pctChange_5m']))
        df_valuation_stock = df_valuation_stock.groupby('c_bondCode').apply(cal_adjust).reset_index()  
        df = pd.merge(df, df_valuation_stock[['c_bondCode','datetime','sec_amplitude_5m']], on=['c_bondCode', 'datetime'], how='left')
        df = df.rename(columns={'datetime': 't_date', 'c_bondCode': 'c_code', 'sec_amplitude_5m': 'value'})
        return factor_pre(df[['t_date', 'c_code', 'value']])

    def ts_amplitude_5m(self, ND):
        # 振幅分位数 ts_amplitude_rollNDtdays_5m
        def cal_adjust(df_slice):
            df_slice = df_slice.copy()
            df_slice['sec_amplitude_5m'][df_slice['tail'] == '09:25:00'] = np.nan
            df_slice = df_slice.sort_values(['datetime'])
            df_slice = df_slice.fillna(method = 'ffill')
            return df_slice
        
        n_trade = self.n_trade
        df = self.df_after[['Date', 'datetime', 'c_bondCode', 'bond_close', 'bond_high', 'bond_low', 'bond_pctChange_5m']].copy()
        df_valuation_stock = self.df_valuation_stock[['Date', 'datetime', 'c_bondCode', 'bond_close', 'bond_high', 'bond_low', 'bond_pctChange_5m','tail']].copy()
        df_valuation_stock['sec_amplitude_5m'] = (df_valuation_stock['bond_high'] - df_valuation_stock['bond_low']) / (df_valuation_stock['bond_close'] / (1 + df_valuation_stock['bond_pctChange_5m']))
        df_valuation_stock = df_valuation_stock.groupby('c_bondCode').apply(cal_adjust).reset_index()  
        df_valuation_stock = df_valuation_stock.set_index('datetime')
        # 滚动ND * n_trade个数据点计算
        min_max_a = df_valuation_stock.groupby('c_bondCode').rolling(ND * n_trade).agg({'sec_amplitude_5m': ['min', 'max']}).reset_index()
        min_max_a.columns = ['c_bondCode', 'datetime', 'min_amplitude_5m', 'max_amplitude_5m']
        df_valuation_stock = pd.merge(df_valuation_stock, min_max_a, on=['c_bondCode', 'datetime'], how='left')
        df_valuation_stock['ts_amplitude_roll{}tdays_5m'.format(ND)] = (df_valuation_stock['sec_amplitude_5m'] - df_valuation_stock['min_amplitude_5m']) / (
                    df_valuation_stock['max_amplitude_5m'] - df_valuation_stock['min_amplitude_5m'])
        df = pd.merge(df, df_valuation_stock[['c_bondCode','datetime','ts_amplitude_roll{}tdays_5m'.format(ND)]], on=['c_bondCode', 'datetime'], how='left')
        df = df.rename(columns={'datetime': 't_date', 'c_bondCode': 'c_code', 'ts_amplitude_roll{}tdays_5m'.format(ND): 'value'})
        return factor_pre(df[['t_date', 'c_code', 'value']])

    def ts_volume_rank_5m(self, ND):
        # N=5 成交量分位数 ts_volume_rollND
        n_trade = self.n_trade
        df = self.df_after[['Date', 'datetime', 'c_bondCode', 'volume']].copy()
        df_valuation_stock = self.df_valuation_stock[['Date', 'datetime', 'c_bondCode', 'volume']].copy()
        df_valuation_stock = df_valuation_stock.set_index('datetime')
        # 滚动ND * n_trade个数据点计算
        rank_n = df_valuation_stock.groupby('c_bondCode').rolling(ND * n_trade).agg({'volume': ['rank', 'count']}).reset_index()
        rank_n.columns = ['c_bondCode', 'datetime', 'volume_rank', 'volume_count']
        df = pd.merge(df, rank_n, on=['c_bondCode', 'datetime'], how='left')
        df['ts_volume_roll{}d'.format(ND)] = df['volume_rank'] /df['volume_count']
        df = df.rename(columns={'datetime': 't_date', 'c_bondCode': 'c_code','ts_volume_roll{}d'.format(ND): 'value'})
        return factor_pre(df[['t_date', 'c_code', 'value']])

    def sec_volume_firm_5m(self, MD):
        # M=5 成交量稳健性 sec_volume_firmNDd_5m
        n_trade = self.n_trade
        df = self.df_after[['Date', 'datetime', 'c_bondCode', 'volume']].copy()
        df_valuation_stock = self.df_valuation_stock[['Date', 'datetime', 'c_bondCode', 'volume']].copy()

        df_valuation_stock = df_valuation_stock.set_index('datetime')
        # 滚动ND * n_trade个数据点计算
        firm = df_valuation_stock.groupby('c_bondCode').rolling(MD * n_trade).agg({'volume': ['mean', 'std']}).reset_index()
        firm.columns = ['c_bondCode', 'datetime', 'volume_mean', 'volume_std']
        df = pd.merge(df, firm, on=['c_bondCode', 'datetime'], how='left')
        df['sec_volume_firm{}d_5m'.format(MD)] = df['volume_mean'] / df['volume_std']
        df = df.rename(columns={'datetime': 't_date', 'c_bondCode': 'c_code','sec_volume_firm{}d_5m'.format(MD): 'value'})
        return factor_pre(df[['t_date', 'c_code', 'value']])

    def sec_ret_5m(self):
        # 收益率 sec_ret_5m
        df = self.df_after[['Date', 'datetime', 'c_bondCode', 'bond_pctChange_5m']].copy()
        df = df.rename(columns={'datetime': 't_date', 'c_bondCode': 'c_code', 'bond_pctChange_5m': 'value'})
        return factor_pre(df[['t_date', 'c_code', 'value']])
    
    def ts_ret_5m(self, ND): # ND = 5
        # 收益率分位数
        n_trade = self.n_trade
        df = self.df_after[['Date', 'datetime', 'c_bondCode', 'bond_pctChange_5m']].copy()
        df_valuation_stock = self.df_valuation_stock[['Date', 'datetime', 'c_bondCode', 'bond_pctChange_5m']].copy()
        df_valuation_stock = df_valuation_stock.set_index('datetime')

        # 滚动ND个数据点计算
        min_max_cp = df_valuation_stock.groupby('c_bondCode').rolling(ND * n_trade).agg({'bond_pctChange_5m': ['min', 'max']}).reset_index()
        min_max_cp.columns = ['c_bondCode', 'datetime', 'min_pctChange', 'max_pctChange']
        df = pd.merge(df, min_max_cp, on=['c_bondCode', 'datetime'], how='left')
        df['ts_ret_roll{}d_5m'.format(ND)] = (df['bond_pctChange_5m'] - df['min_pctChange']) / (df['max_pctChange'] - df['min_pctChange'])
        df = df.rename(columns={'datetime': 't_date', 'c_bondCode': 'c_code', 'ts_ret_roll{}d_5m'.format(ND): 'value'})
        return factor_pre(df[["t_date", "c_code", "value"]])

    def sec_price_ratio_5m(self, MD):
        # M=5 价格强弱因子 sec_price_ratioMd_5m
        n_trade = self.n_trade
        df = self.df_after[['Date', 'datetime', 'c_bondCode', 'bond_close']].copy()
        df_valuation_stock = self.df_valuation_stock[['Date', 'datetime', 'c_bondCode', 'bond_close']].copy()

        df_valuation_stock = df_valuation_stock.set_index('datetime')
        # 滚动ND * n_trade个数据点计算
        pr = df_valuation_stock.groupby('c_bondCode').rolling(MD * n_trade).bond_close.mean().reset_index()
        pr.columns = ['c_bondCode', 'datetime', 'close_mean']
        df = pd.merge(df, pr, on=['c_bondCode', 'datetime'], how='left')
        df['sec_price_ratio_5m{}D'.format(MD)] = df['bond_close'] / df['close_mean'] - 1

        df = df.rename(columns={'datetime': 't_date', 'c_bondCode': 'c_code',
                                'sec_price_ratio_5m{}D'.format(MD): 'value'})
        return factor_pre(df[['t_date', 'c_code', 'value']])

    def sec_conv_premium_5m(self):
        # 转股溢价率 sec_conv_premium_5m
        df = self.df_after[['Date', 'datetime', 'c_bondCode', 'n_convPrice', 'bond_close', 'stock_close']].copy()
        df['convValue_5m'] = 100 / df['n_convPrice'] * df['stock_close']
        df['sec_conv_premium_5m'] = df['bond_close'] / df['convValue_5m'] - 1
        df = df.rename(columns={'datetime': 't_date', 'c_bondCode': 'c_code', 'sec_conv_premium_5m': 'value'})
        return factor_pre(df[['t_date', 'c_code', 'value']])

    def ts_conv_premium_5m(self, ND):
        # 转股溢价率分位数 ts_conv_premium_rollNDtdays_5m
        n_trade = self.n_trade
        df = self.df_after[['Date', 'datetime', 'c_bondCode', 'n_convPrice', 'bond_close', 'stock_close']].copy()
        df['convValue_5m'] = 100 / df['n_convPrice'] * df['stock_close']
        df['sec_conv_premium_5m'] = df['bond_close'] / df['convValue_5m'] - 1
        df_valuation_stock = self.df_valuation_stock[
            ['Date', 'datetime', 'c_bondCode', 'n_convPrice', 'bond_close', 'stock_close']].copy()
        df_valuation_stock['convValue_5m'] = 100 / df_valuation_stock['n_convPrice'] * df_valuation_stock['stock_close']
        df_valuation_stock['sec_conv_premium_5m'] = df_valuation_stock['bond_close'] /\
                                                    df_valuation_stock['convValue_5m'] - 1

        df_valuation_stock = df_valuation_stock.set_index('datetime')
        # 滚动ND * n_trade个数据点计算
        min_max_cp = df_valuation_stock.groupby('c_bondCode').rolling(ND * n_trade).agg(
            {'sec_conv_premium_5m': ['min', 'max']}).reset_index()
        min_max_cp.columns = ['c_bondCode', 'datetime', 'min_convPremium_5m', 'max_convPremium_5m']
        df = pd.merge(df, min_max_cp, on=['c_bondCode', 'datetime'], how='left')
        df['ts_conv_premium_roll{}tdays_5m'.format(ND)] = (df['sec_conv_premium_5m'] - df['min_convPremium_5m']) / (
                    df['max_convPremium_5m'] - df['min_convPremium_5m'])

        df = df.rename(columns={'datetime': 't_date', 'c_bondCode': 'c_code',
                                'ts_conv_premium_roll{}tdays_5m'.format(ND): 'value'})
        return factor_pre(df[['t_date', 'c_code', 'value']])

    def sec_modified_premium_5m(self):
        # 修正溢价率 sec_modified_premium_5m
        def modified_premium_cal(df_slice):
            # 计算修正溢价率
            y_data = df_slice.convPremium_5m
            df_slice.loc[:, 'convValue_5m'] = df_slice['convValue_5m'].replace(0, 0.00001)
            x_data = sm.add_constant(1 / df_slice.convValue_5m)
            a, b = sm.OLS(y_data, x_data).fit().params
            new_x_data = a + b * x_data.iloc[:, 1]
            df_slice.loc[:, 'sec_modified_premium_5m'] = df_slice['convPremium_5m'] - new_x_data
            return df_slice

        df = self.df_after[['Date', 'datetime', 'c_bondCode', 'n_convPrice', 'bond_close', 'stock_close']].copy()
        df['convValue_5m'] = 100 / df['n_convPrice'] * df['stock_close']
        df['convPremium_5m'] = df['bond_close'] / df['convValue_5m'] - 1
        tqdm.pandas(desc='apply')
        df = df.groupby('datetime', as_index=False).progress_apply(modified_premium_cal).reset_index(drop=True)
        df = df.rename(columns={'datetime': 't_date', 'c_bondCode': 'c_code', 'sec_modified_premium_5m': 'value'})
        return factor_pre(df[['t_date', 'c_code', 'value']])

    def ts_modified_premium_5m(self, ND):
        # 修正溢价率分位数 ts_modified_premium_rollNDtdays_5m
        def get_conv_data(df_slice, n, trade):
            # 获得历史转股价值和转股溢价率数据
            df_slice = df_slice.reset_index(drop=True)
            length = len(df_slice)
            if len(df_slice) < n * trade:
                lst1, lst2 = [np.nan] * length, [np.nan] * length
            else:
                lst1, lst2 = [np.nan] * (n * trade - 1), [np.nan] * (n * trade - 1)
                lst_convValue = df_slice['convValue_5m'].values
                lst_convPremium = df_slice['convPremium_5m'].values
                for k in range(n * trade - 1, length):
                    lst1.append(lst_convValue[k - (n * trade - 1):k - (n * trade - 1) + n * trade])
                    lst2.append(lst_convPremium[k - (n * trade - 1):k - (n * trade - 1) + n * trade])
            df_slice['all_convValue_5m'] = lst1
            df_slice['all_convPremium_5m'] = lst2
            return df_slice[['datetime', 'c_bondCode', 'all_convValue_5m', 'all_convPremium_5m']]

        def modified_premium_cal(df_slice, n):
            # 计算修正溢价率分位数
            def conv_ols(x, y):
                # ols回归
                x = sm.add_constant(1 / x)
                a, b = sm.OLS(y, x).fit().params
                new_x = a + b * x.T[1]
                return y - new_x

            convValue = np.array([list(df_slice['all_convValue_5m'].values[i]) for i in range(len(df_slice))]).T
            convValue[convValue == 0] = 0.00001
            convPremium = np.array([list(df_slice['all_convPremium_5m'].values[i]) for i in range(len(df_slice))]).T

            all_mp = np.array(list(map(conv_ols, convValue, convPremium))).T
            max_mp, min_mp = np.max(all_mp, axis=1), np.min(all_mp, axis=1)
            # 计算当期分位数
            df_slice['ts_modified_premium_roll{}tdays_5m'.format(n)] = list(map(lambda x, y, z: (x - y) / (z - y),
                                                                                all_mp.T[-1], min_mp, max_mp))
            df_slice['ts_modified_premium'] = list(all_mp.T[-1])
            df_slice['min'] = min_mp
            df_slice['max'] = max_mp
            return df_slice

        n_trade = self.n_trade
        df = self.df_after[['Date', 'datetime', 'c_bondCode', 'n_convPrice', 'bond_close', 'stock_close']].copy()
        df_valuation_stock = self.df_valuation_stock[
            ['Date', 'datetime', 'c_bondCode', 'n_convPrice', 'bond_close', 'stock_close']].copy()
        df_valuation_stock['convValue_5m'] = 100 / df_valuation_stock['n_convPrice'] * df_valuation_stock['stock_close']
        df_valuation_stock['convPremium_5m'] = df_valuation_stock['bond_close'] / df_valuation_stock['convValue_5m'] - 1

        tqdm.pandas(desc='apply')
        # 获取历史20天的回归数据
        ols_data = df_valuation_stock.groupby('c_bondCode').progress_apply(
            lambda x: get_conv_data(x, ND, n_trade)).reset_index(drop=True)
        df = pd.merge(df, ols_data, on=['c_bondCode', 'datetime'], how='left')
        # 除去历史数据不满20天的
        df = df.dropna(axis=0, subset=['all_convValue_5m']).reset_index(drop=True)
        # 计算修正溢价率及其分位数
        df = df.groupby('datetime').progress_apply(lambda x: modified_premium_cal(x, ND)).reset_index(drop=True)
        df = df.rename(columns={'datetime': 't_date', 'c_bondCode': 'c_code',
                                'ts_modified_premium_roll{}tdays_5m'.format(ND): 'value'})
        return factor_pre(df[['t_date', 'c_code', 'value']])

    def sec_double_low_5m(self):
        # 双低 sec_double_low_5m
        df = self.df_after[['Date', 'datetime', 'c_bondCode', 'n_convPrice', 'bond_close', 'stock_close']].copy()
        df['convValue_5m'] = 100 / df['n_convPrice'] * df['stock_close']
        df['convPremium_5m'] = df['bond_close'] / df['convValue_5m'] - 1
        df['sec_double_low_5m'] = df['convPremium_5m'] * df['bond_close']
        df = df.rename(columns={'datetime': 't_date', 'c_bondCode': 'c_code', 'sec_double_low_5m': 'value'})
        return factor_pre(df[['t_date', 'c_code', 'value']])

    def ts_double_low_5m(self, ND):
        # 双低分位数 ts_double_low_rollNDtdays_5m
        n_trade = self.n_trade
        df = self.df_after[['Date', 'datetime', 'c_bondCode', 'n_convPrice', 'bond_close', 'stock_close']].copy()
        df['convValue_5m'] = 100 / df['n_convPrice'] * df['stock_close']
        df['convPremium_5m'] = df['bond_close'] / df['convValue_5m'] - 1
        df['sec_double_low_5m'] = df['convPremium_5m'] * df['bond_close']
        df_valuation_stock = self.df_valuation_stock[
            ['Date', 'datetime', 'c_bondCode', 'n_convPrice', 'bond_close', 'stock_close']].copy()
        df_valuation_stock['convValue_5m'] = 100 / df_valuation_stock['n_convPrice'] * df_valuation_stock['stock_close']
        df_valuation_stock['convPremium_5m'] = df_valuation_stock['bond_close'] / df_valuation_stock['convValue_5m'] - 1
        df_valuation_stock['sec_double_low_5m'] = df_valuation_stock['convPremium_5m'] *\
                                                  df_valuation_stock['bond_close']

        df_valuation_stock = df_valuation_stock.set_index('datetime')
        # 滚动ND * n_trade个数据点计算
        min_max_dl = df_valuation_stock.groupby('c_bondCode').rolling(ND * n_trade).agg(
            {'sec_double_low_5m': ['min', 'max']}).reset_index()
        min_max_dl.columns = ['c_bondCode', 'datetime', 'min_double_low_5m', 'max_double_low_5m']
        df = pd.merge(df, min_max_dl, on=['c_bondCode', 'datetime'], how='left')
        df['ts_double_low_roll{}tdays_5m'.format(ND)] = (df['sec_double_low_5m'] - df['min_double_low_5m']) / (
                    df['max_double_low_5m'] - df['min_double_low_5m'])

        df = df.rename(columns={'datetime': 't_date', 'c_bondCode': 'c_code',
                                'ts_double_low_roll{}tdays_5m'.format(ND): 'value'})
        return factor_pre(df[['t_date', 'c_code', 'value']])

    def sec_amplitude_delta_5m(self):
        # 振幅偏差 sec_amplitude_delta_5m
        def cal_adjust(df_slice):
            df_slice = df_slice.copy()
            df_slice['bond_amplitude_5m'][df_slice['tail'] == '09:25:00'] = np.nan
            df_slice['stock_amplitude_5m'][df_slice['tail'] == '09:25:00'] = np.nan
            df_slice = df_slice.sort_values(['datetime'])
            df_slice = df_slice.fillna(method = 'ffill')
            return df_slice
        
        df = self.df_after[['Date', 'datetime', 'c_bondCode', 'bond_close', 'bond_high', 'bond_low', 'stock_close',
                            'stock_high', 'stock_low', 'bond_pctChange_5m', 'stock_pctChange_5m']].copy()
        df_valuation_stock = self.df_valuation_stock[['Date', 'datetime', 'c_bondCode', 'bond_close', 'bond_high', 'bond_low', 'stock_close',
                            'stock_high', 'stock_low', 'bond_pctChange_5m', 'stock_pctChange_5m','tail']].copy()
        df_valuation_stock['bond_amplitude_5m'] = (df_valuation_stock['bond_high'] - df_valuation_stock['bond_low']) / (
            df_valuation_stock['bond_close'] / (1 + df_valuation_stock['bond_pctChange_5m']))
        df_valuation_stock['stock_amplitude_5m'] = (df_valuation_stock['stock_high'] - df_valuation_stock['stock_low']) / (
            df_valuation_stock['stock_close'] / (1 + df_valuation_stock['stock_pctChange_5m']))
        df_valuation_stock = df_valuation_stock.groupby('c_bondCode').apply(cal_adjust).reset_index()  
        df_valuation_stock['sec_amplitude_delta_5m'] = df_valuation_stock['stock_amplitude_5m'] - df_valuation_stock['bond_amplitude_5m']
        df = pd.merge(df, df_valuation_stock[['c_bondCode','datetime','sec_amplitude_delta_5m']], on=['c_bondCode', 'datetime'], how='left')
        df = df.rename(columns={'datetime': 't_date', 'c_bondCode': 'c_code', 'sec_amplitude_delta_5m': 'value'})
        return factor_pre(df[['t_date', 'c_code', 'value']])

    def sec_ret_delta_5m(self):
        # 涨跌幅差 sec_ret_delta_5m
        df = self.df_after[['Date', 'datetime', 'c_bondCode', 'bond_pctChange_5m', 'stock_pctChange_5m']].copy()
        df['sec_price_delta_5m'] = df['bond_pctChange_5m'] - df['stock_pctChange_5m']
        df = df.rename(columns={'datetime': 't_date', 'c_bondCode': 'c_code', 'sec_price_delta_5m': 'value'})
        return factor_pre(df[['t_date', 'c_code', 'value']])

    def ts_ret_delta_5m(self, ND):
        # 涨跌幅差分位数 ts_ret_delta_rollNDtdays_5m
        n_trade = self.n_trade
        df = self.df_after[['Date', 'datetime', 'c_bondCode', 'bond_pctChange_5m', 'stock_pctChange_5m']].copy()
        df['sec_price_delta_5m'] = df['bond_pctChange_5m'] - df['stock_pctChange_5m']
        df_valuation_stock = self.df_valuation_stock[
            ['Date', 'datetime', 'c_bondCode', 'bond_pctChange_5m', 'stock_pctChange_5m']].copy()
        df_valuation_stock['sec_price_delta_5m'] = df_valuation_stock['bond_pctChange_5m'] -\
                                                   df_valuation_stock['stock_pctChange_5m']

        df_valuation_stock = df_valuation_stock.set_index('datetime')
        # 滚动ND * n_trade个数据点计算
        min_max_dl = df_valuation_stock.groupby('c_bondCode').rolling(ND * n_trade).agg(
            {'sec_price_delta_5m': ['min', 'max']}).reset_index()
        min_max_dl.columns = ['c_bondCode', 'datetime', 'min_price_delta_5m', 'max_price_delta_5m']
        df = pd.merge(df, min_max_dl, on=['c_bondCode', 'datetime'], how='left')
        df['ts_price_delta_roll{}tdays_5m'.format(ND)] = (df['sec_price_delta_5m'] - df['min_price_delta_5m']) / (
                    df['max_price_delta_5m'] - df['min_price_delta_5m'])

        df = df.rename(columns={'datetime': 't_date', 'c_bondCode': 'c_code',
                                'ts_price_delta_roll{}tdays_5m'.format(ND): 'value'})
        return factor_pre(df[['t_date', 'c_code', 'value']])


def df_factor_data(fn, start, end, method):
    """
    通过函数提取因子数据
    """
    lst = {
           'sec_bond_premium_5m': "cfc.sec_bond_premium_5m()",                      # 纯债溢价率
           'ts_bond_premium_roll5d_5m': "cfc.ts_bond_premium_5m(ND=5)",             # 纯债溢价率5日分位数
           'sec_convprice_5m': "cfc.sec_price_5m()",                                # 转债价格
           'ts_convprice_roll20d_5m': "cfc.ts_price_5m(ND=20)",                     # 转债价格20日分位数
           'ts_convprice_roll5d_5m': "cfc.ts_price_5m(ND=5)",                       # 转债价格5日分位数
           'sec_weightskew5d_5m': "cfc.sec_price_skew_5m(MD=5)",                    # 5日转债价格偏度因子
           'sec_convturnover_5m': "cfc.sec_turnover_5m()",                          # 换手率
           'ts_convturnover_roll5d_5m': "cfc.ts_turnover_5m(ND=5)",                 # 换手率5日分位数
           'sec_volatility5d_5m': "cfc.sec_volatility_5m(MD=5)",                    # 5日波动率
           'ts_volatility1h_roll1d_5m': "cfc.ts_volatility_5m(MD=0.25,ND=1)",       # 1h波动率1日分位数
           'sec_amplitude_5m': "cfc.sec_amplitude_5m()",                            # 振幅
           'ts_amplitude_roll5d_5m': "cfc.ts_amplitude_5m(ND=5)",                   # 振幅5日分位数
           'ts_volumerank_roll5d_5m': "cfc.ts_volume_rank_5m(ND=5)",                # 成交量5日分位数
           'sec_volstability_5d_5m': "cfc.sec_volume_firm_5m(MD=5)",                # 10日成交量稳健性
           'sec_ret_5m': "cfc.sec_ret_5m()",                                        # 当前收益率
           'sec_priceratio_5d_5m': "cfc.sec_price_ratio_5m(MD=5)",                  # 5日价格强弱
           'sec_convpremium_5m': "cfc.sec_conv_premium_5m()",                       # 转股溢价率
           'ts_conv_premium_roll5d_5m': "cfc.ts_conv_premium_5m(ND=5)",             # 转股溢价率5日分位数
           'sec_modified_premium_5m': "cfc.sec_modified_premium_5m()",              # 修正溢价率
           'ts_modified_premium_roll5d_5m': "cfc.ts_modified_premium_5m(ND=5)",     # 修正溢价率5日分位数
           'sec_double_low_5m': "cfc.sec_double_low_5m()",                          # 双低
           'ts_double_low_roll5d_5m': "cfc.ts_double_low_5m(ND=5)",                 # 双低5日分位数
           'sec_amplitude_delta_5m': "cfc.sec_amplitude_delta_5m()",                # 振幅偏差
           'sec_ret_delta_5m': "cfc.sec_ret_delta_5m()",                            # 涨跌幅差
           'ts_ret_delta_roll5d_5m': "cfc.ts_ret_delta_5m(ND=5)",                   # 涨跌幅差5日分位数
           'ts_ret_roll5d_5m': "cfc.ts_ret_5m(ND=5)"                                # 收益率5日分位数
            }
    
    # 计算并存储因子
    file_path = './output/因子值/'
    factor = None  # 初始化 factor 变量
    if method == 'update':
        factor_saved = pd.read_pickle(os.path.join(file_path, '{}.pkl').format(fn))
        factor_saved = factor_saved.dropna(subset=['value'])
        start = pd.to_datetime(factor_saved.t_date.unique()[-1]) + pd.DateOffset(days=1)
        start = start.strftime('%Y-%m-%d')
        cfc = ConvFactorCalculation(start, end)
        #if pd.to_datetime(start) >= pd.to_datetime(end):
        if cfc.judgement() == 0:
            print('{}_因子已经更新到最新'.format(fn))
        else:
            print('计算因子: {}'.format(fn))
            cal_time = time.time()
            #cfc = ConvFactorCalculation(start, end)
            factor = eval(lst[fn])
            factor['factor_name'] = fn
            factor = pd.concat([factor_saved, factor],axis=0)
            print('计算用时: {:.2f}s'.format(time.time() - cal_time))
            csv_time = time.time()
            factor = factor.sort_values(['t_date', 'c_code'])
            factor = factor.drop_duplicates(['t_date','c_code'])
            factor.to_pickle(os.path.join(file_path, '{}.pkl').format(fn))
            print('导出用时: {:.2f}s'.format(time.time() - csv_time))

    elif method == 'history_supple': 
        factor_saved = pd.read_pickle(os.path.join(file_path, '{}.pkl').format(fn))
        end = pd.to_datetime(factor_saved.t_date.unique()[0]) - pd.DateOffset(days=1)
        end = end.strftime('%Y-%m-%d')
        if pd.to_datetime(start) >= pd.to_datetime(end):
            print('{}_因子已经历史数据已补齐'.format(fn))
        else:
            print('计算因子: {}'.format(fn))
            cal_time = time.time()
            cfc = ConvFactorCalculation(start, end)
            factor = eval(lst[fn])
            factor['factor_name'] = fn
            factor = pd.concat([factor_saved, factor],axis=0)
            print('计算用时: {:.2f}s'.format(time.time() - cal_time))
            csv_time = time.time()
            factor = factor.sort_values(['t_date', 'c_code'])
            factor = factor.drop_duplicates(['t_date','c_code'])
            factor.to_pickle(os.path.join(file_path, '{}.pkl').format(fn))
            print('导出用时: {:.2f}s'.format(time.time() - csv_time))
        
    elif method == 'recover':
        start = start
        print('计算因子: {}'.format(fn))
        cal_time = time.time()
        cfc = ConvFactorCalculation(start, end)
        factor = eval(lst[fn])
        factor['factor_name'] = fn
        print('计算用时: {:.2f}s'.format(time.time() - cal_time))
        csv_time = time.time()
        factor = factor.sort_values(['t_date', 'c_code'])
        factor = factor.drop_duplicates(['t_date','c_code'])
        factor.to_pickle(os.path.join(file_path, '{}.pkl').format(fn))
        print('导出用时: {:.2f}s'.format(time.time() - csv_time))
    return factor


if __name__ == '__main__':
    start = '2022-06-30'
    #end = '2024-11-02'
    end = str(date.today())
    factor_names = [
                    'sec_amplitude_5m',
                    'sec_amplitude_delta_5m',
                    'sec_bond_premium_5m',
                    'sec_convpremium_5m',
                    'sec_convprice_5m',
                    'sec_convturnover_5m',
                    'sec_double_low_5m',
                    'sec_modified_premium_5m',
                    'sec_priceratio_5d_5m',
                    'sec_ret_5m',
                    'sec_ret_delta_5m',
                    'sec_volatility5d_5m',
                    'sec_volstability_5d_5m',
                    'sec_weightskew5d_5m',
                    'ts_amplitude_roll5d_5m',
                    'ts_bond_premium_roll5d_5m',
                    'ts_conv_premium_roll5d_5m',
                    'ts_convprice_roll20d_5m',
                    'ts_convprice_roll5d_5m',
                    'ts_convturnover_roll5d_5m',
                    'ts_double_low_roll5d_5m',
                    'ts_modified_premium_roll5d_5m',
                    'ts_ret_delta_roll5d_5m',
                    'ts_ret_roll5d_5m',
                    'ts_volatility1h_roll1d_5m',
                    'ts_volumerank_roll5d_5m'
                    ]

    factors = []
    for fn in factor_names:
        """
        method = update 更新提取数据
        method = history_supple 补充历史数据
        method = recover 重新提取数据(覆盖历史)
        """
        factor_data = df_factor_data(fn, start, end, method='update')
        factors.append(factor_data)